\documentclass{beamer}
    \usetheme[height=7mm]{Rochester}
    \usepackage{listings}
    \usepackage{algorithmic}
    \usepackage{hyperref}

    \title{Hamal}
    \subtitle{GUI Test Automation Solution for Mac OS X}
    \author{Ajax Wang \\ajwang@cisco.com}
    \date{April 23, 2012}

    \begin{document}
        \maketitle
        \begin{frame}
            \frametitle{Before we start}
            Automation tools that OS X supports:
            \begin{enumerate}
                \item AppleScript (Editor)
                \item Scripting Bridge(since 10.5)
                \item Automator(since 10.4)
            \end{enumerate}

            \vspace{0.5cm}
            What we need:
            \begin{enumerate}
                \item A tool that can be controlled programmably, and
                \item will perform GUI automation actions
            \end{enumerate}

            \vspace{0.5cm}
            \textbf{Hamal} uses something equivalent to AppleScript.
        \end{frame}



        \begin{frame}
            \frametitle{AppleScript}
            \begin{block}{AppleScript}
                \textbf{AppleScript} is a scripting language that makes possible direct control of
                \emph{scriptable applications} and of many parts of the Mac OS.\\
            \end{block}


            \begin{block}{Scriptable Application}
                A scriptable application is one that makes its operations and data available in response to AppleScript messages, called Apple events.\\
            \end{block}

            With scriptable applications, users can write scripts to automate operations, while developers can use AppleScript as an aid to rapid prototyping and automated testing.\\
        \end{frame}



        \begin{frame}
            \frametitle{An AppleScript Example}
            Type into \textbf{AppleScript Editor} the following snippet and hit the \textbf{run} button then:
            \begin{algorithmic}[1]
                \STATE \textbf{tell} \textit{application} "Cisco Jabber" \textbf{to} \textbf{launch}\\
                \vspace{0.3cm}
                \STATE \textbf{delay} 5\\
                \vspace{0.3cm}
                \STATE \textbf{tell} \textit{application} "System Events" \\
                \STATE \hspace{0.5cm}\textbf{tell} \textit{process} "Cisco Jabber" \\
                \STATE \hspace{1.0cm}\textbf{click} \textit{button} "Sign In" \textbf{of}
                    \textit{window} "Cisco Jabber" \\
                \STATE \hspace{0.5cm}\textbf{end tell} \\
                \STATE \textbf{end tell} \\
            \end{algorithmic}
        \end{frame}



        \begin{frame}
            \frametitle{The Underlying Mechanism}
            The \textbf{Open Scripting Architecture} plays the key role.
            \begin{center}
              \includegraphics[height=6.0cm]{as-arch.pdf}
            \end{center}
        \end{frame}


        \begin{frame}
            \frametitle{What if an application is not scriptable?}
            Our Mac IM client, \textbf{Cisco Jabber}(A.K.A \textbf{Aries}),
            is not scriptable, responses to no Apple Events.\\

            \vspace{0.5cm}
            \textbf{Q}: How are its tests automated? \\
            \textbf{A}: GUI scripting(in System Events). \\

            \vspace{0.5cm}
            \begin{block}{GUI scripting}
                \textbf{GUI scripting} allows your scripts to perform some actions in applications that have no built-in scripting support.

                \vspace{0.3cm}
                GUI control is performed by writing scripts incorporating terms and commands from the Processes Suite in the \textbf{System Events} application's scripting dictionary.
            \end{block}
        \end{frame}


        \begin{frame}
            \frametitle{More on GUI scripting}
            \begin{block}{How it works}
            GUI Scripting is based upon the Mac OS X \textit{Accessibility Frameworks} that provide alternative methods of querying and controlling the interfaces of the OS and applications.
            \end{block}
            \vspace{0.3cm}
            So we should check \textbf{Eable access for assistive device} option before using it.
        \end{frame}


        \begin{frame}
            \frametitle{Limitations \& Our Corresponding Solutions}

            \begin{block}{Limitations}
            \begin{enumerate}
              \item OS X only
              \item text processing \& math operations are challenging
            \end{enumerate}
            \end{block}


            \begin{block}{Our thoughts}
            \begin{enumerate}
              \item agent, client/server model
              \item use AppScript instade of AppleScript
              \item use other language(e.g. python)
            \end{enumerate}
            \end{block}

        \end{frame}



        \begin{frame}
            \frametitle{Appscript}
            Python implementation of Applescript\\
            Offical site: \url{http://appscript.sourceforge.net/}\\
            \vspace{0.5cm}

            Equivalent \textbf{Python} code as above \textbf{AppleScript} sample:\\
            \begin{algorithmic}[1]
            \STATE app(u'Cisco Jabber').launch(waitreply=False)\\
            \STATE time.sleep(5)\\
            \STATE app(u'System Events').processes[u'Cisco Jabber'].windows[u'Cisco Jabber'].buttons[u'Sign In'].click()
            \end{algorithmic}
        \end{frame}



        \begin{frame}
            \frametitle{Hamal}

            \begin{block}{Overall description}
                \begin{enumerate}
                  \item Hosted on Mac OS X, a faceless background application
                  \item perform GUI actions in response to clients' requests
                  \item Has direct access to the host OS
                  \item Distributed, any where your network extend to
                \end{enumerate}
            \end{block}

            \begin{block}{Core componets}
                \begin{enumerate}
                  \item Python lib
                  \item Thrift
                  \item Appscript
                  \item Our Automation API(Wrapper functions)
                \end{enumerate}
            \end{block}
        \end{frame}



        \begin{frame}
            \frametitle{Architecture}
            \begin{figure}[h]
            \centering
            {
                \includegraphics[width=8cm]{hamal-arche.pdf}
            }
            \end{figure}

        \end{frame}


        \begin{frame}
            \frametitle{Data exchange}

            Client send to Server Commands of form \textbf{action, target, value}
            \begin{itemize}
              \item action tells Hamal server what user action to perform, e.g. click left button
              \item target specify where the action is going to perform at
              \item value gives additional information action needs, like what to type into textbox
            \end{itemize}

            And in turn Server will return the executing result back to Client.

            \vspace{0.3cm}
            You may conceive two messages no more than a stirng.

        \end{frame}



        \begin{frame}
            \frametitle{Our wrapper functions classification criteria}

            Users interact with PC by means of I/O devices:
            \begin{itemize}
              \item mouse
              \item keyboard
              \item screen
              \item \ldots
            \end{itemize}

            So all what we can do while using PC is:
            \begin{itemize}
              \item click some place on screen
              \item read messages that our friends send to us from MSN
              \item type some character into chat box and click \textbf{Enter} to send
            \end{itemize}

            Thus, our basic APIs fall into 3 categories: mouse op, keystroke op and screen reading.
        \end{frame}



        \begin{frame}
            \frametitle{Our basic wrapper functions}
            Mouse click:
            \begin{itemize}
              \item leftclick
              \item rightclick
              \item doubleclick
            \end{itemize}

            Keyboard input:
            \begin{itemize}
              \item keycode
              \item keystroke
              \item combined keystroke/keycode
            \end{itemize}

            Screen info extracte:
            \begin{itemize}
              \item getXXX, XXX can be title, value, \ldots
            \end{itemize}
        \end{frame}


        \begin{frame}
            \frametitle{Miscellaneous}
            \begin{block}{System level mouse click}
            Mouse click issued by GUI scripting is not system level, I mean not the same as a human click.
            a) The event is post to application directly, OS has no aware of this and thus won't move mouse
            according to the element click event aimed at. b) Moveover, clicks to no handler elements are ignored.

            \vspace{0.3cm}
            So we call C API to send low-level mouse events.
            \end{block}


            \begin{block}{Auto front-most}
            While performing some actions, like click, the target application
            should be on top most(should not be overlapped by other windows),
            otherwise the following actions will be incorrect.

            \vspace{0.3cm}
            We need bring application window to top most before every action.
            \end{block}
        \end{frame}
\end{document} 